{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.layers import Input, Dense, Lambda, Layer, Activation\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from keras import metrics, optimizers\n",
    "from keras.callbacks import Callback\n",
    "import keras\n",
    "\n",
    "from keras.utils import plot_model\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Function for reparameterization trick to make model differentiable\n",
    "def sampling(args):\n",
    "    \n",
    "    import tensorflow as tf\n",
    "    # Function with args required for Keras Lambda function\n",
    "    z_mean, z_log_var = args\n",
    "\n",
    "    # Draw epsilon of the same shape from a standard normal distribution\n",
    "    epsilon = K.random_normal(shape=tf.shape(z_mean), mean=0.,\n",
    "                              stddev=epsilon_std)\n",
    "    \n",
    "    # The latent vector is non-deterministic and differentiable\n",
    "    # in respect to z_mean and z_log_var\n",
    "    z = z_mean + K.exp(z_log_var / 2) * epsilon\n",
    "    return z\n",
    "\n",
    "\n",
    "class CustomVariationalLayer(Layer):\n",
    "    \"\"\"\n",
    "    Define a custom layer that learns and performs the training\n",
    "    This function is borrowed from:\n",
    "    https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder.py\n",
    "    \"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        # https://keras.io/layers/writing-your-own-keras-layers/\n",
    "        self.is_placeholder = True\n",
    "        super(CustomVariationalLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def vae_loss(self, x_input, x_decoded):\n",
    "        reconstruction_loss = original_dim * metrics.binary_crossentropy(x_input, x_decoded)\n",
    "        kl_loss = - 0.5 * K.sum(1 + z_log_var_encoded - K.square(z_mean_encoded) - \n",
    "                                K.exp(z_log_var_encoded), axis=-1)\n",
    "        return K.mean(reconstruction_loss + (K.get_value(beta) * kl_loss))\n",
    "\n",
    "    def call(self, inputs):\n",
    "        x = inputs[0]\n",
    "        x_decoded = inputs[1]\n",
    "        loss = self.vae_loss(x, x_decoded)\n",
    "        self.add_loss(loss, inputs=inputs)\n",
    "        # We won't actually use the output.\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class WarmUpCallback(Callback):\n",
    "    def __init__(self, beta, kappa):\n",
    "        self.beta = beta\n",
    "        self.kappa = kappa\n",
    "    # Behavior on each epoch\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if K.get_value(self.beta) <= 1:\n",
    "            K.set_value(self.beta, K.get_value(self.beta) + self.kappa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10459, 5000)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RPS4Y1</th>\n",
       "      <th>XIST</th>\n",
       "      <th>KRT5</th>\n",
       "      <th>AGR2</th>\n",
       "      <th>CEACAM5</th>\n",
       "      <th>KRT6A</th>\n",
       "      <th>KRT14</th>\n",
       "      <th>CEACAM6</th>\n",
       "      <th>DDX3Y</th>\n",
       "      <th>KDM5D</th>\n",
       "      <th>...</th>\n",
       "      <th>FAM129A</th>\n",
       "      <th>C8orf48</th>\n",
       "      <th>CDK5R1</th>\n",
       "      <th>FAM81A</th>\n",
       "      <th>C13orf18</th>\n",
       "      <th>GDPD3</th>\n",
       "      <th>SMAGP</th>\n",
       "      <th>C2orf85</th>\n",
       "      <th>POU5F1B</th>\n",
       "      <th>CHST2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>0.678296</td>\n",
       "      <td>0.289910</td>\n",
       "      <td>0.034230</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.084731</td>\n",
       "      <td>0.031863</td>\n",
       "      <td>0.037709</td>\n",
       "      <td>0.746797</td>\n",
       "      <td>0.687833</td>\n",
       "      <td>...</td>\n",
       "      <td>0.440610</td>\n",
       "      <td>0.428782</td>\n",
       "      <td>0.732819</td>\n",
       "      <td>0.634340</td>\n",
       "      <td>0.580662</td>\n",
       "      <td>0.294313</td>\n",
       "      <td>0.458134</td>\n",
       "      <td>0.478219</td>\n",
       "      <td>0.168263</td>\n",
       "      <td>0.638497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>0.200633</td>\n",
       "      <td>0.654917</td>\n",
       "      <td>0.181993</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.100606</td>\n",
       "      <td>0.050011</td>\n",
       "      <td>0.092586</td>\n",
       "      <td>0.103725</td>\n",
       "      <td>0.140642</td>\n",
       "      <td>...</td>\n",
       "      <td>0.620658</td>\n",
       "      <td>0.363207</td>\n",
       "      <td>0.592269</td>\n",
       "      <td>0.602755</td>\n",
       "      <td>0.610192</td>\n",
       "      <td>0.374569</td>\n",
       "      <td>0.722420</td>\n",
       "      <td>0.271356</td>\n",
       "      <td>0.160465</td>\n",
       "      <td>0.602560</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 5000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   RPS4Y1      XIST      KRT5  AGR2  CEACAM5     KRT6A  \\\n",
       "TCGA-02-0047-01  0.678296  0.289910  0.034230   0.0      0.0  0.084731   \n",
       "TCGA-02-0055-01  0.200633  0.654917  0.181993   0.0      0.0  0.100606   \n",
       "\n",
       "                    KRT14   CEACAM6     DDX3Y     KDM5D    ...      FAM129A  \\\n",
       "TCGA-02-0047-01  0.031863  0.037709  0.746797  0.687833    ...     0.440610   \n",
       "TCGA-02-0055-01  0.050011  0.092586  0.103725  0.140642    ...     0.620658   \n",
       "\n",
       "                  C8orf48    CDK5R1    FAM81A  C13orf18     GDPD3     SMAGP  \\\n",
       "TCGA-02-0047-01  0.428782  0.732819  0.634340  0.580662  0.294313  0.458134   \n",
       "TCGA-02-0055-01  0.363207  0.592269  0.602755  0.610192  0.374569  0.722420   \n",
       "\n",
       "                  C2orf85   POU5F1B     CHST2  \n",
       "TCGA-02-0047-01  0.478219  0.168263  0.638497  \n",
       "TCGA-02-0055-01  0.271356  0.160465  0.602560  \n",
       "\n",
       "[2 rows x 5000 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnaseq_file = os.path.join('data', 'pancan_scaled_zeroone_rnaseq.tsv.gz')\n",
    "rnaseq_df = pd.read_table(rnaseq_file, index_col=0)\n",
    "print(rnaseq_df.shape)\n",
    "rnaseq_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split 10% test set randomly\n",
    "test_set_percent = 0.1\n",
    "rnaseq_test_df = rnaseq_df.sample(frac=test_set_percent)\n",
    "rnaseq_train_df = rnaseq_df.drop(rnaseq_test_df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set hyper parameters\n",
    "original_dim = rnaseq_df.shape[1]\n",
    "latent_dim = 100\n",
    "\n",
    "batch_size = 50\n",
    "epochs = 50\n",
    "learning_rate = 0.0005\n",
    "\n",
    "epsilon_std = 1.0\n",
    "beta = K.variable(0)\n",
    "kappa = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input place holder for RNAseq data with specific input size\n",
    "rnaseq_input = Input(shape=(original_dim, ))\n",
    "\n",
    "# Input layer is compressed into a mean and log variance vector of size `latent_dim`\n",
    "# Each layer is initialized with glorot uniform weights and each step (dense connections,\n",
    "# batch norm, and relu activation) are funneled separately\n",
    "# Each vector of length `latent_dim` are connected to the rnaseq input tensor\n",
    "z_mean_dense_linear = Dense(latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input)\n",
    "z_mean_dense_batchnorm = BatchNormalization()(z_mean_dense_linear)\n",
    "z_mean_encoded = Activation('relu')(z_mean_dense_batchnorm)\n",
    "\n",
    "z_log_var_dense_linear = Dense(latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input)\n",
    "z_log_var_dense_batchnorm = BatchNormalization()(z_log_var_dense_linear)\n",
    "z_log_var_encoded = Activation('relu')(z_log_var_dense_batchnorm)\n",
    "\n",
    "# return the encoded and randomly sampled z vector\n",
    "# Takes two keras layers as input to the custom sampling function layer with a `latent_dim` output\n",
    "z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean_encoded, z_log_var_encoded])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The decoding layer is much simpler with a single layer and sigmoid activation\n",
    "decoder_to_reconstruct = Dense(original_dim, kernel_initializer='glorot_uniform', activation='sigmoid')\n",
    "rnaseq_reconstruct = decoder_to_reconstruct(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 5000)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 100)          500100      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 100)          500100      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 100)          400         dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 100)          400         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 100)          0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 100)          0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 100)          0           activation_1[0][0]               \n",
      "                                                                 activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5000)         505000      lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "custom_variational_layer_1 (Cus [(None, 5000), (None 0           input_1[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 1,506,000\n",
      "Trainable params: 1,505,600\n",
      "Non-trainable params: 400\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "adam = optimizers.Adam(lr=learning_rate)\n",
    "vae_layer = CustomVariationalLayer()([rnaseq_input, rnaseq_reconstruct])\n",
    "vae = Model(rnaseq_input, vae_layer)\n",
    "vae.compile(optimizer=adam, loss=None, loss_weights=[beta])\n",
    "\n",
    "vae.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 8s, sys: 10.2 s, total: 2min 18s\n",
      "Wall time: 1min 29s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hist = vae.fit(np.array(rnaseq_train_df),\n",
    "               shuffle=True,\n",
    "               epochs=epochs,\n",
    "               verbose=0,\n",
    "               batch_size=batch_size,\n",
    "               validation_data=(np.array(rnaseq_test_df), None),\n",
    "               callbacks=[WarmUpCallback(beta, kappa)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "[Errno 2] No such file or directory: 'figures/onehidden_vae_training.pdf'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIOError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-6d6e1120b252>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_ylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'VAE Loss'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist_plot_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/liy0f/.conda/envs/tf_newest_py2/lib/python2.7/site-packages/matplotlib/figure.pyc\u001b[0m in \u001b[0;36msavefig\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1571\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_frameon\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframeon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1572\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1573\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1574\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1575\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mframeon\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/liy0f/.conda/envs/tf_newest_py2/lib/python2.7/site-packages/matplotlib/backend_bases.pyc\u001b[0m in \u001b[0;36mprint_figure\u001b[0;34m(self, filename, dpi, facecolor, edgecolor, orientation, format, **kwargs)\u001b[0m\n\u001b[1;32m   2250\u001b[0m                 \u001b[0morientation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morientation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2251\u001b[0m                 \u001b[0mbbox_inches_restore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_bbox_inches_restore\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2252\u001b[0;31m                 **kwargs)\n\u001b[0m\u001b[1;32m   2253\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2254\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbbox_inches\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mrestore_bbox\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/liy0f/.conda/envs/tf_newest_py2/lib/python2.7/site-packages/matplotlib/backends/backend_pdf.pyc\u001b[0m in \u001b[0;36mprint_pdf\u001b[0;34m(self, filename, **kwargs)\u001b[0m\n\u001b[1;32m   2517\u001b[0m             \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2518\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2519\u001b[0;31m             \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPdfFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2520\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2521\u001b[0m             \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwidth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/liy0f/.conda/envs/tf_newest_py2/lib/python2.7/site-packages/matplotlib/backends/backend_pdf.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename)\u001b[0m\n\u001b[1;32m    420\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell_base\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    421\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_string_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 422\u001b[0;31m             \u001b[0mfh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    423\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mis_writable_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: 'figures/onehidden_vae_training.pdf'"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEKCAYAAADq59mMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt4VdWd//H3N8lJTu4hEEJIggEBEaGAosV6qaX1btXeRGutrR2dqq3aOp2qbX+2HW2ntaO1U6ttta12tMpUq9baohVEnaoVEEGuAnIJ13AJCYTcv78/9iYcQkISyMkJyef1POc5+6x9OWtD4JO1195rmbsjIiLSFUmJroCIiBx5FB4iItJlCg8REekyhYeIiHSZwkNERLpM4SEiIl2m8BARkS5TeIiISJcpPEREpMtSEl2BeBk0aJCXlZUluhoiIkeMuXPnbnX3gs5s22fDo6ysjDlz5iS6GiIiRwwzW9PZbXXZSkREukzhISIiXabwEBGRLuuzfR4i0v80NDRQXl5ObW1toqvSq0WjUUpKSohEIod8DIWHiPQZ5eXlZGdnU1ZWhpklujq9kruzbds2ysvLGT58+CEfJ26Xrcwsamb/NLN3zGyRmX0vLB9uZm+a2Qoze8LMUsPytPDzinB9Wcyxbg3Ll5nZ2fGqs4gc2Wpraxk4cKCC4yDMjIEDBx526yyefR51wFR3nwBMBM4xsynAj4B73H0ksAP4Urj9l4AdYfk94XaY2VjgUuA44BzgF2aWHMd6i8gRTMHRse74M4pbeHhgV/gxEr4cmAr8MSx/GLg4XL4o/Ey4/qMWnOFFwOPuXufu7wMrgJPiVe+fvfQes5dXxOvwIiJ9QlzvtjKzZDObD2wBXgRWApXu3hhuUg4Uh8vFwDqAcP1OYGBseRv7dLtfzl7Jy8u2xOvwItLHZWVlJboKPSKu4eHuTe4+ESghaC2Mief3mdk1ZjbHzOZUVBxa6yE7GqG6trHjDUVE+rEeec7D3SuBWcDJQJ6Z7b3LqwRYHy6vB0oBwvW5wLbY8jb2af09v3L3ye4+uaCgU8OzHCA7mkJ1bcMh7Ssispe7841vfINx48Yxfvx4nnjiCQA2btzI6aefzsSJExk3bhyvvvoqTU1NfOELX2jZ9p577klw7TsWt1t1zawAaHD3SjNLB84k6ASfBXwaeBy4Engm3OXZ8PPr4fqZ7u5m9izwmJndDQwFRgH/jFe9g/BQy0PkSPe9Py9i8Yaqbj3m2KE53P7x4zq17VNPPcX8+fN555132Lp1KyeeeCKnn346jz32GGeffTbf+ta3aGpqoqamhvnz57N+/XreffddACorK7u13vEQz+c8ioCHwzujkoDp7v6cmS0GHjezO4C3gYfC7R8Cfm9mK4DtBHdY4e6LzGw6sBhoBK5396Z4VTo7GmH77vp4HV5E+onXXnuNyy67jOTkZAoLC/nwhz/MW2+9xYknnshVV11FQ0MDF198MRMnTmTEiBGsWrWKr371q5x//vmcddZZia5+h+IWHu6+AJjURvkq2rhbyt1rgc+0c6w7gTu7u45tyY6msHZ7TU98lYjEUWdbCD3t9NNP55VXXuEvf/kLX/jCF/j617/O5z//ed555x1mzJjBAw88wPTp0/nNb36T6KoelMa2aiXoMFefh4gcntNOO40nnniCpqYmKioqeOWVVzjppJNYs2YNhYWFXH311fzLv/wL8+bNY+vWrTQ3N/OpT32KO+64g3nz5iW6+h3S8CSt5ERTqFKfh4gcpk984hO8/vrrTJgwATPjxz/+MUOGDOHhhx/mrrvuIhKJkJWVxSOPPML69ev54he/SHNzMwA//OEPE1z7jik8WsmOplDf2ExdYxNpKXqQXUS6Zteu4NloM+Ouu+7irrvu2m/9lVdeyZVXXnnAfkdCayOWLlu1kh0NRpnUHVciIu1TeLSSHQ0aYwoPEZH2KTxa2dfyUKe5iEh7FB6tqOUhItIxhUcr+8JDLQ8RkfYoPFrJCS9b6XZdEZH2KTxa0WUrEZGOKTxayUrTZSsR6RkHm/tj9erVjBs3rgdr0zUKj1ZSkpPISE1Wy0NE5CD0hHkbNKeHSB/w11tg08LuPeaQ8XDuf7a7+pZbbqG0tJTrr78egO9+97ukpKQwa9YsduzYQUNDA3fccQcXXXRRl762traWa6+9ljlz5pCSksLdd9/NRz7yERYtWsQXv/hF6uvraW5u5sknn2To0KFccskllJeX09TUxHe+8x2mTZt2WKfdFoVHGzSboIgcimnTpnHTTTe1hMf06dOZMWMGN9xwAzk5OWzdupUpU6Zw4YUXYmadPu59992HmbFw4UKWLl3KWWedxfLly3nggQe48cYbufzyy6mvr6epqYnnn3+eoUOH8pe//AWAnTt3xuVcFR5t0IRQIn3AQVoI8TJp0iS2bNnChg0bqKioYMCAAQwZMoSvfe1rvPLKKyQlJbF+/Xo2b97MkCFDOn3c1157ja9+9asAjBkzhqOOOorly5dz8sknc+edd1JeXs4nP/lJRo0axfjx47n55pv55je/yQUXXMBpp50Wl3NVn0cbNCy7iByqz3zmM/zxj3/kiSeeYNq0aTz66KNUVFQwd+5c5s+fT2FhIbW1td3yXZ/97Gd59tlnSU9P57zzzmPmzJmMHj2aefPmMX78eL797W/z/e9/v1u+qzW1PNqQHU2hXBNCicghmDZtGldffTVbt25l9uzZTJ8+ncGDBxOJRJg1axZr1qzp8jFPO+00Hn30UaZOncry5ctZu3YtxxxzDKtWrWLEiBHccMMNrF27lgULFjBmzBjy8/P53Oc+R15eHg8++GAczlLh0SbN6SEih+q4446jurqa4uJiioqKuPzyy/n4xz/O+PHjmTx5MmPGjOnyMa+77jquvfZaxo8fT0pKCr/73e9IS0tj+vTp/P73vycSiTBkyBBuu+023nrrLb7xjW+QlJREJBLh/vvvj8NZgrl7XA6caJMnT/Y5c+Yc0r4/eH4JD/9jNcvuOLebayUi8bRkyRKOPfbYRFfjiNDWn5WZzXX3yZ3ZX30ebchOS6GusZn6xuZEV0VEpFfSZas2xA6OODArLcG1EZG+bOHChVxxxRX7laWlpfHmm28mqEadE7fwMLNS4BGgEHDgV+5+r5lNAB4AsoDVwOXuXmVmZcASYFl4iDfc/cvhsU4AfgekA88DN3ocr7dlxcwmqPAQObK4e5eeoUi08ePHM3/+/B79zu747zOel60agZvdfSwwBbjezMYCDwK3uPt44E/AN2L2WenuE8PXl2PK7weuBkaFr3PiWG8NjihyhIpGo2zbtq1b/nPsq9ydbdu2EY1GD+s4cWt5uPtGYGO4XG1mS4BiYDTwSrjZi8AM4DvtHcfMioAcd38j/PwIcDHw13jVXXN6iByZSkpKKC8vp6KiItFV6dWi0SglJSWHdYwe6fMIL0lNAt4EFgEXAU8DnwFKYzYdbmZvA1XAt939VYLAKY/Zpjwsi5u9c3pU16nlIXIkiUQiDB8+PNHV6BfifreVmWUBTwI3uXsVcBVwnZnNBbKB+nDTjcAwd58EfB14zMxyuvhd15jZHDObczi/eeiylYjIwcU1PMwsQhAcj7r7UwDuvtTdz3L3E4A/ACvD8jp33xYuzw3LRwPrgdj2VUlYdgB3/5W7T3b3yQUFBYdc7+yWDnNdthIRaUvcwsOC2x0eApa4+90x5YPD9yTg2wR3XmFmBWaWHC6PIOgYXxX2nVSZ2ZTwmJ8HnolXvUEtDxGRjsSzz+MU4ApgoZntvQ/tNmCUmV0ffn4K+G24fDrwfTNrAJqBL7v79nDddey7VfevxLGzHCCSnEQ0kqSWh4hIO+J5t9VrQHs3W9/bxvZPElziautYc4AenY9Rc3qIiLRPw5O0Q3N6iIi0T+HRjuxohCpdthIRaZPCox05anmIiLRL4dGO4LKVWh4iIm1ReLQjO00d5iIi7VF4tEMd5iIi7VN4tCM7GmFPQxMNTZoQSkSkNYVHO/Y+Zb5LrQ8RkQMoPNqhIUpERNqn8GjH3sER9ayHiMiBFB7tyFHLQ0SkXQqPdmhYdhGR9ik82qE+DxGR9ik82qF5zEVE2qfwaMe+y1ZqeYiItKbwaEdqShJpKUlU1yk8RERaU3gcRDAhlC5biYi0pvA4iJxoClW6bCUicgCFx0FkR1M0PImISBsUHgehy1YiIm1TeByEhmUXEWmbwuMgstIUHiIibYlbeJhZqZnNMrPFZrbIzG4MyyeY2etmttDM/mxmOTH73GpmK8xsmZmdHVN+Tli2wsxuiVedW9NlKxGRtsWz5dEI3OzuY4EpwPVmNhZ4ELjF3ccDfwK+ARCuuxQ4DjgH+IWZJZtZMnAfcC4wFrgs3DbusqMp7K5voqnZe+LrRESOGHELD3ff6O7zwuVqYAlQDIwGXgk3exH4VLh8EfC4u9e5+/vACuCk8LXC3Ve5ez3weLht3GlCKBGRtvVIn4eZlQGTgDeBRez7z/8zQGm4XAysi9mtPCxrr7yt77nGzOaY2ZyKiorDrneO5vQQEWlT3MPDzLKAJ4Gb3L0KuAq4zszmAtlAfXd9l7v/yt0nu/vkgoKCwz6eRtYVEWlbSjwPbmYRguB41N2fAnD3pcBZ4frRwPnh5uvZ1woBKAnLOEh5XGlODxGRtsXzbisDHgKWuPvdMeWDw/ck4NvAA+GqZ4FLzSzNzIYDo4B/Am8Bo8xsuJmlEnSqPxuvesdSy0NEpG3xbHmcAlwBLDSz+WHZbQRBcH34+SngtwDuvsjMpgOLCe7Uut7dmwDM7CvADCAZ+I27L4pjvVu0hEedWh4iIrHiFh7u/hpg7ay+t5197gTubKP8eeD57qtd52hODxGRtukJ84PQZSsRkbYpPA4iGkkmNTlJt+qKiLSi8OiABkcUETmQwqMDCg8RkQMpPDqgwRFFRA6k8OiAWh4iIgdSeHQgCA+1PEREYik8OhBctlLLQ0QklsKjA7psJSJyIIVHB7KjEXbVNWpCKBGRGAqPDuTsnRCqTq0PEZG9OgwPMzvFzDLD5c+Z2d1mdlT8q9Y77BuiRJ3mIiJ7dablcT9QY2YTgJuBlcAjca1VL7J3cES1PERE9ulMeDS6uxNMHftzd7+PYAbAfkGDI4qIHKgzQ7JXm9mtwOeA08NJnCLxrVbvodkERUQO1JmWxzSgDviSu28imAb2rrjWqhdRy0NE5ECdankA97p7Uzjn+BjgD/GtVu+xNzyqFB4iIi060/J4BUgzs2LgBYKpZX8Xz0r1Jjm6bCUicoDOhIe5ew3wSeAX7v4ZYFx8q9V7pKUkEUk2XbYSEYnRqfAws5OBy4G/dGG/PsHMNCy7iEgrnQmBm4BbgT+5+yIzGwHMim+1epesNI1vJSISq8PwcPfZ7n4hcJ+ZZbn7Kne/oaP9zKzUzGaZ2WIzW2RmN4blE83sDTObb2ZzzOyksPwMM9sZls83s/8Xc6xzzGyZma0ws1sO43wPiQZHFBHZX4d3W5nZeIInyvODj1YBfN7dF3WwayNws7vPM7NsYK6ZvQj8GPieu//VzM4LP58R7vOqu1/Q6vuTgfuAM4Fy4C0ze9bdF3f6LA+T5vQQEdlfZy5b/RL4ursf5e7DCIYo+XVHO7n7RnefFy5XA0uAYsCBnHCzXGBDB4c6CVgRtnjqgccJnnbvfo318MxXYOEf9yvWnB4iIvvrzHMeme7e0sfh7i/vHSixs8ysDJgEvEnQhzLDzH5CEF4fitn0ZDN7hyBQ/i1s3RQD62K2KQc+2JXv77SUVHjvRWiqh/GfbinWZSsRkf11puWxysy+Y2Zl4evbwKrOfoGZZQFPAje5exVwLfA1dy8FvgY8FG46DzjK3ScA/w083ZUTCb/rmrAfZU5FRUVXdw8MnQgb5u9XlBONUKXLViIiLToTHlcBBcBTBCEwCPhiZw5uZpFwn0fd/amw+MrwWAD/S3BZCnevcvdd4fLzQMTMBgHrgdKYw5aEZQdw91+5+2R3n1xQUNCZKh6oaCJsXQ51u1qKsqMp7KprpFkTQomIAJ2722qHu9/g7se7+wnufhPw7Y72MzMjaFUscfe7Y1ZtAD4cLk8F3gu3HxLuQ3gHVhKwDXgLGGVmw80sFbgUeLbTZ9hVQycCDpsWthRlR1Nwh931unQlIgKd6/NoyyXAv3WwzSkEQ5ksNLO914FuA64G7jWzFKAWuCZc92ngWjNrBPYAl4ZDwTea2VeAGUAy8JtO3Ol16IomBu8b58NRJwOxI+s2tiyLiPRnhxoe1tEG7v7aQbY7oY3tfw78vJ1jPQ8835UKHrKcIsgqhI3vtBRpZF0Rkf21Gx5mlt/eKjoRHke0ov07zTWnh4jI/g7W8phL8ExGW0FRH5/q9BJDJ8KKF6F+N6RmquUhItJKu+Hh7sN7siK9StEE8GbY9C4M+yA5LXN6qOUhIgL9aHTcLontNGf/DnMREVF4tC1nKGQWtPR76LKViMj+FB5tMQtaH+EdV+mRZJKTTB3mIiKhdsPDzKbGLA9vte6T8axUrzB0IlQshYY94YRQGt9KRGSvg7U8fhKz/GSrdR0+YX7EK5oI3hR0mrNviBIRETl4eFg7y2197nuGtuo0T9NUtCIiex0sPLyd5bY+9z05xZAxqKXTfEBmhM1VdQmulIhI73CwhwRHmNmzBK2MvcuEn/v+MyBmwfMeYcvjxLJ87n3pPbbtqmNgVlqCKyciklgHC4/Y2fp+0mpd689909CJ8NpPoWEPU8cM5qd/f4+Xl1XwqRNKEl0zEZGEOtgT5rPbKjezUoJh0dtc36fs7TTfvJhxQ4+nIDuNmcu2KDxEpN/r1HMeZlZgZteZ2avAy0BhXGvVW7R0mr9NUpLxkWMKeGVZBQ1NzYmtl4hIgh3sOY9sM7vSzGYA/wSOBoa7+9Hu3tFcHn1Dbimk57d0mk8dU0h1XSNzVu9IcMVERBLrYC2PLQRT0N4BjHD3m+nro+m2Zha0PsJO81NHDSKSbMxatiXBFRMRSayDhcetQBrwC+BWMzu6Z6rUyxRNhC1LoKGWrLQUPjh8IC8t2ZzoWomIJFS74eHuP3X3Key76+ppYKiZfdPMRvdI7XqDoROhuRG2BDPfTh0zmJUVu1mzbXeCKyYikjgddpi7+yp3/4G7jwcmAzn01JSwvUHRhOC9pd9jMAAzl+rSlYj0XwfrML/PzE6NLXP3d939W+4+Mv5V6yXyjoJoXssIu2WDMhlRkKnwEJF+7WAtj+XAXWa22sx+bGaTeqpSvUqrTnOAqccM5s1V29mtgRJFpJ86WJ/Hve5+MvBhYBvwGzNbama3d6bPw8xKzWyWmS02s0VmdmNYPtHM3jCz+WY2x8xOCsvNzH5mZivMbIGZHR9zrCvN7L3wdeVhn3VXFU2EzYuhMRjbauqxg6lvaua1FVt7vCoiIr1BZ/o81rj7j9x9EnAZcDGwpBPHbgRudvexwBTgejMbC/wY+J67TwT+X/gZ4FxgVPi6BrgfwMzygduBDwInAbeb2YDOn2I3GDoRmhtgy2IgGOcqOy2FWbp0JSL9VIfhYWYpZvZxM3sU+CuwDOhwMih33+ju88LlaoLAKSYYkTcn3CwX2BAuXwQ84oE3gDwzKwLOBl509+3uvgN4ETinKyd52PbOaR52mkeSkzht9CBmLt2Ce98fYFhEpLV2x7YyszMJWhrnETxh/jhwjbt3+R5VMysDJgFvAjcBM8zsJwTh9aFws2JgXcxu5WFZe+U9Z0BZ2Gke0+8xppDnF25i0YYqxhXn9mh1REQSraOHBP8BHOvuF7r7Y4cYHFkEMxHe5O5VwLXA19y9FPga8NAh1Lu977om7EeZU1FR0V2HDTrNh50MC5+EjQsAOOOYAszgpSW6dCUi/c/BOsynuvuD4aWiQ2JmEYLgeNTdnwqLrwT2Lv8vQT8GwHqgNGb3krCsvfK26vwrd5/s7pMLCgoOtdptu+BuiObCo5+GHWsYlJXGhJI8ZmqoEhHphzo1qu6hMDMjaFUscfe7Y1ZtILiDC2Aq8F64/Czw+fCuqynATnffCMwAzjKzAWFH+VlhWc/KGQqfexIaa+F/PgU125k6ZjALyiupqNYMgyLSv8QtPIBTgCuAqeFtufPN7DzgauC/zOwd4AcEd1ZB8NT6KmAF8GvgOgB33w78B/BW+Pp+WNbzBo+Byx6HyrXwh0v56Mgc3OFltT5EpJ+xvnq30OTJk33OnDnxOfjiZ2D6lfgx5/KhVV9gUtkgfnH5CfH5LhGRHmJmc919cme2jWfLo+8aexGc+2Ns2fPck/MYs5dtYcfu/jVavYj0bwqPQ/XBa+CUm5iy7Wmuan6Sn89akegaiYj0GIXH4fjo7fCBadyc8r/kv/lj1mmYdhHpJxQehyMpCS76BTXjPsv1yX9i7SPXQJMGSxSRvk/hcbiSU8j41C94o/gLnLLzOXY+8lloqE10rURE4krh0R3MOO6Kn/Bj+yK5a2bg//MJ2FOZ6FqJiMSNwqObZEcjFJ55EzfUfwVf9xb87nyo3pToaomIxIXCoxtddtIwFgz4GN+Kfhvf/j48dBbUJOZ5RhGReFJ4dKPUlCT+/Zwx/GH7KGae+EvYuQ5e/a9EV0tEpNspPLrZueOGMGlYHrfNSadx/KXwz19D5bqOdxQROYIoPLqZmXHbeceyuaqOR9M/CzjM/s9EV0tEpFspPOLgxLJ8zhpbyF1v1LBrwhdh/mNQsSzR1RIR6TYKjzi57bxjAfjy6g/jkQyY+R8JrpGISPdReMRJ2aBMfjptIv+3Ef6W8xlY8mcon5voaomIdAuFRxx9bGwh/3bWMdxcfhp7IgPg77dDHx0CX0T6F4VHnF13xtF8ZPxwflRzIax+FVbOTHSVREQOm8IjzsyMuz7zAeYVXEw5BdTNuB2amxNdLRGRw6Lw6AEZqSn84sop/NIuJa1iITXvPJnoKomIHBaFRw8pGZDB+ZffwLLmUqr+cjtNDZp5UESOXAqPHjRl5GA2TP4GQxrX89SDd1LfqMtXInJkUnj0sDMuuILy3BP46KYHuf6hl6iqbUh0lUREukzh0cMsKYmSy+5lgNXwofIHueSB19m4c0+iqyUi0iVxCw8zKzWzWWa22MwWmdmNYfkTZjY/fK02s/lheZmZ7YlZ90DMsU4ws4VmtsLMfmZmFq9694gh47ETruTKlL+TuuM9PnHfP1i6qSrRtRIR6bR4tjwagZvdfSwwBbjezMa6+zR3n+juE4Engadi9lm5d527fzmm/H7gamBU+DonjvXuGVO/TVJqJo+VPoN7M5+5/3X+sWJromslItIpcQsPd9/o7vPC5WpgCVC8d33YergE+MPBjmNmRUCOu7/h7g48Alwcr3r3mMxBcMY3ySqfzfPn7aEoL8qVv/0nv39jDc3NegpdRHq3HunzMLMyYBLwZkzxacBmd38vpmy4mb1tZrPN7LSwrBgoj9mmnJgQavU915jZHDObU1FR0W31j5sTr4aBoxj42vf436snM2XEQL7z9Ltc+us3WFmxK9G1ExFpV9zDw8yyCC5P3eTusRf2L2P/VsdGYJi7TwK+DjxmZjld+S53/5W7T3b3yQUFBYdb9fhLSYVzfgjbVpC74Lc8ctVJ/OhT41m6sYpz732Vn898T7fzikivFNfwMLMIQXA86u5PxZSnAJ8Enthb5u517r4tXJ4LrARGA+uBkpjDloRlfcOoM2HkmTD7R9jurUw7cRh/v/nDnHlsIT95YTkX/vw13l67I9G1FBHZTzzvtjLgIWCJu9/davXHgKXuXh6zfYGZJYfLIwg6xle5+0agysymhMf8PPBMvOqdEGf/ABpqYNYdAAzOjnLf5cfz689PprKmgU/e/w++++widu7RMyEi0jvEs+VxCnAFMDXm9tvzwnWXcmBH+enAgvDW3T8CX3b37eG664AHgRUELZK/xrHePa9gNJz0rzD3YdjwdkvxmWMLefHrp3PFlKN4+PXVTP3Jy0x/a5061EUk4cz76PwSkydP9jlz5iS6Gp23pxJ+Phnqd8OpX4cPfQUi6S2r312/k9ufXcTcNTuYUJrH9y88jgmleQmssIj0NWY2190nd2pbhUcvsmM1vPAdWPIs5A6Ds/4Dxl4E4TOR7s6f3l7PD55fyrbdtXx5fAr/+qEi8somJLbeItInKDw4QsNjr/dfgb/dCpvfhaNODe7IKvoA7FgDq1+jYeVsapbPJrd+EwDPF3yJvLNuZcrRg0hKOrIfvheRxFF4cISHB0BTI8x7GGbeAXt2QE4xVIX3F2QMhLJT2TLoJMrfmc3xO1/gj02n87P067ng+DI+eXwJIwdnJbb+InLEUXjQB8Jjrz074NW7oXINHHUKlJ0GBWMgKbzXwZ2GmT8k8uqPWJw2kc9WXUelZzGhNI+rTinj/PFFpCRr/EsR6ZjCgz4UHp31zhPwzPU05pXxxzH38OuFTays2E1xXjpXnzacS04sJSM1JdG1FJFeTOFBPwwPgNWvweOXQ1IKzZf+gZm7juKB2SuZs2YHeRkRPn9yGVeefBQDs9ISXVMR6YUUHvTT8ADY+h48+mmo3gQfuQ2mXMecddX88pVVvLh4M9FIEueNL+LTx5cwZcRAdbCLSAuFB/04PAB2b4VnvwrLnofBx8EF98CwD7JiSzUPvbaa597ZQHVdI0Nzo3zi+GI+eXwJRxeog12kv1N40M/DY68lz8Ff/x2q1sMJX4CP3g4Z+dQ2NPHC4s08ObecV9+roNlhYmkeF04YytnjhlCcl97hoUWk71F4oPBoUbcLXv4hvHE/pA8IxtH6wCUtDx5uqarl6fnreWreepZuqgZgXHEOZ48dwjnjhjBycBZH+sSNItI5Cg8UHgfYuACeuwnWzw2eGRlzfvA66hRIjgCwqmIXMxZt5oXFm3h7bSUAIwZl8rGxhZwxuoDJZfmkpui2X5G+SuGBwqNNzU3w7pOw6GlY+RI01kI0F0afEwRJ2WmQkQ/A5qpaXli8mRcWbeKNVdtoaHIyUpP50NGD+PAxBZwxuoDS/IwEn5CIdCeFBwqPDtXvhpWzYOlzsOyvUBu0NMgugsFjYfCxUHgcDD6WXbmjeH3NbmYv38LLyyoo37EHgBE6VOHNAAASmklEQVQFmUw9ZjBTjx3MiWX5RPQwosgRTeGBwqNLmhph7evBJa0tS2DLYqhYBk11wfr0fDjtZjjxX/CUNN7fupvZyyuYuXQLb67aTn1TM9lpKZw+uoCPjBnMGccUMEjPkogccRQeKDwOW1Mj7HgfNi8KxthaORNySuCMW2DCZZAcPK2+u66R/1uxlZlLtzBz6Ra2VNdhBscNzeHUkQWcOnIQk8sGEI0kJ/iERKQjCg8UHt1u1Wx46XtB62TQaJj6HTj24y13bQE0NzuLN1Yxc+kWXntvK/PW7qCx2UlLSeKk4flhkORz3NAchYlIL6TwQOERF+5BH8lL/wFbl8HAkZBZAEkpwSs5Ei4ngzuNjY1U7t7Dzt217NxdS219A+/6cF7kJOoLj2fCsHwmluYxoTSP4QMz9bS7SIIpPFB4xFVTIyx4PLhrq6ku+NzcAM2N4XIjWFIQIknJYMF7Q2MjyZsWkOQNbEsaxN+aTuDPDSfyVvMxpKelMWZINscW5TCmKHwfkq3BHEV6kMIDhUevVbsTls+Axc/gK17CGvdQm5rPwuzTeNZP5ZltpVTVNQPBFbHhAzOZOCyPScMGMKk0jzFDsjXEvEicKDxQeBwR6nfDey8G0+4u+ys01OC5pVSPupiF+Wczp2YIC9fvZP66HWzdVQ9ANJLEB4rzmFCay6jCbI4uyGLk4Cxy0yMJPhmRI5/CA4XHEaduVzCQ44LpwZ1d3gSF42HM+XjBMWyOlDJ3Vz5zN9Tx9rodLNpQRX1jc8vuBdlpHF2QycjBWYwvzuUDJXmMGpylVopIF/SK8DCzUuARoBBw4Ffufq+ZPQEcE26WB1S6+8Rwn1uBLwFNwA3uPiMsPwe4F0gGHnT3/+zo+xUeR7BdFbDoqSBI1rf6O8wpgYFH0zxwJJVpJayzISytL2B+dS5LtzWwYvMuqusaAUiPJIdBksuE0jxGFWZRMiCDrDT1o4i0pbeERxFQ5O7zzCwbmAtc7O6LY7b5L2Cnu3/fzMYCfwBOAoYCfwdGh5suB84EyoG3gMtij9MWhUcfUV8D21fBtvdg2wrYumLfcu3OmA0NcorxAWXsSh/Keh/Esj15zKvK4v+2ZrC2cQD1BJe2BmREKM3PoHRABiX56YwYlMmYITmMLswmPTU5uJy2OfzxKj2x589ZJEG6Eh5x+xXM3TcCG8PlajNbAhQDi8NKGnAJMDXc5SLgcXevA943sxUEQQKwwt1Xhfs9Hm570PCQPiI1A4aMC16t1WyH7e8H4bIjeLftq8he/xpjqjcyBuciCH7KU2BPdDCVqUVsSipkbf1Alq8dyLtLclnWXM8aW8O6pLVMSFlLsW8kieCXql3DzyHtwp8QGVDak2ct0uv1SPvdzMqAScCbMcWnAZvd/b3wczHwRsz68rAMYF2r8g/GpaJyZMnID14lJxy4rrE+mMdkZznsXAeV60ivXEt65RqKKpcyqXp90K8S8y9gZ9pQVqWM4O/1p/N6TREj2MiNq56i7qeT+WX0c7xbfAkjBucyoiCLQVmp5GemMiAjeM9ITdbQ9dKvxD08zCwLeBK4yd2rYlZdRnCZqju/6xrgGoBhw4Z156HlSJOSCvnDg1dbmhqDcKlcEzzYWHgcudFcJhH8ljOtvpH3Nu9i9uovMHrOd/nKzgdZuvpl/n3ZVfyiqeyAw6WmJDEwM5WygZkcMySb0YXZHDMki1GF2eREdSeY9D1xvdvKzCLAc8AMd787pjwFWA+c4O7lYdmtAO7+w/DzDOC74S7fdfez29quPerzkG7jHgxl/7db8ZqtVI+7gh3Zx1DVlEZlcxqVDRG2NqSyqTaVt3bmsGxLDbvrm1p2L8qNMqIgk2H5GZTmZzAs5pWbHlGLRXqNXtHnEfZpPAQsiQ2O0MeApXuDI/Qs8JiZ3U3QYT4K+CdgwCgzG04QOJcCn41XvUUOYAbjPw0jP4r9/XvkzP0dObTzS1daLn7sqVQWncKyjON5e3cByzZXs3pbDS8s2sy23fX7bZ4TTeGogZkcNTCDsoGZDAvfS/PTGZSV1vlh7mu2B3OzJGnMMOkZ8bzb6lTgVWAhsPeG/Nvc/Xkz+x3whrs/0GqfbwFXAY0El7n+GpafB/yU4Fbd37j7nR19v1oeEjf1u4M7vep3Q1011O8KnlPZsx3WvhEMIrlzbbBtTjEM/3Bw19bg49iVO4p1NSms3V7D2m01rN1ew+ptu1m3tZr0qpWMYwWTbAUDrJoXmibzeurJZGbnMjArjYKsNAZlpVKUl05xXjrFuamM2PEauQt+i73/Mgw9Hj7xABQcc9Dqi7SnV9yqm2gKD0kY9+Dur1UvB6/3X4E9O/atzymBwnDCLSwYqXjD20EIAY2RbOqSM8is3Uy9pfFO5in8PfJhZjaMY9OuJqy2kkuSZ/P55BcYllTBRs/n1dTTOLdpFlGv5c3h17Nt/FWUDsymZEA6AzPTSNagk9IJCg8UHtKLNDcHd3xtWQJbFoXvS4IJtyC4Dbl4MpRMhuITIP/o4FLZujdhwROw6E9B+GQMhGEn4ytnYg017Cg4kXeGTuON1Cmsq2ygevsGrtp2D2cwhzebx/BvDf/KOi8EICXJiEaSSUtJannPjqYwJDfKkJwoQ3LTKcqNtnwuyE4jUw9T9jsKDxQecgRoagBvhpQOZl1srIcVf4eF02HNP2DUWfDBf4Uh4w/c1p26eY+SMuMWvLmJt0Z9jbcGXkxtYzO1Dc3UNTZR29BMbWMTVXsa2LSzlk1VtVTXNh5wqMzUZAqy0/a9stIoygtCpjgvnaF56QzOTtMQMH2IwgOFh/RzO8vhmeuDy2ZpuUFneloWpGXve0XzgjnrswvZEx3MVgawsTmPdbXpVOxuYGt1LVuq69haXUtFdR1bdtWzs7Z5v69JMijMiZKbHiEtkkx6JGjZpEeSiUaSyUhNJjsaITuaQk40pWU5Nz3CgMxU8jNSyU2PaC6XXqJX3G0lIgmUWwJXPA3vPA4b5gUd+3tfNdthx5qgg79mGwDpQGn4Oukgh/WcTBrTBlAbyWNXUg6VZLO1OYtdzanU1qVQW5vMnqZkapqT2d2UTEVTJu/W5bO6aRA7ySS4eXJ/SUbLw5YDMlLJSY+QE01p9R6hMDdKcV6Uotx0XVLrBfQ3INJXmcHEy4JXexrrYfcWqN4UvjYG4RJ7jGABvBmr3UmkZhuRPdvJrtlGUc3GIIQa9kBTfZtfQSR4Nadm05BdQm1mKbtT86lrdGr3vhqaqW1spro6ifKdg1jVVMC8+kEsqxtAnR/4kGVueoSheekU50XJy0glMzWZjLSU4D01hcy0ZCLJSS3VtzC0zCCSnMSAjNSWUQLyMlJ1Q8EhUHiI9GcpqUErJbfk8I/lHgRIY92+95qtQSuncg1JlWtJ27GGtMq15G57O9jemwEPlvFgn8bafYdMMzynmPrsYdREBrDTstnWnMWWhgw21KeztiLK1vpUNjQks6M+hermFOo8lT2kUkOUZjruj2nd8snLiATvmRHy0lMZlNZEVmYWuWGrKDc9Qk56hOy0lH59uU3hISLdwyzo/I+9ASC3GIomdP4Y7rBrS3Cr847V2Pb3sR3vE92xhmj1MvJrtjN8zw5o6yHNVg0UtySa0gfRlFFAU8ZgmjIKaMwYTF10EDuTB1Fh+WxsHsD6xlwq9jSzbVc9NbuqyN4ynyF7ljKyYTnjWMnwpM2sbC7iueaT+XPTFFZ4Scvp5mektrqpIMLIpM3keRVVpFPVnEGlp1PVlEZNg1PX2NRS89ju5iSDgVlpDMmJUrj3DricKINz0ohGeueDn+owF5EjS3Mz1FYGty/XbA+ej2nYA417oKE2fN8TPMi5a3MQRi2vzdDc0OqABpkFwU0EO94PW0PgOcU0DZlITd4okte9ScbGNzCcyuxRrCj4GAtyp1JRa+RsW8jg6sUcVbeUY5pWkG17DqwyRg3p7LA8FiUfw8KksSxIHkt5UjGY0djczNbqevY0NB2wb0qSkZJsRJKTSE1OIpKcRCRl3+fUlJj3JGd4dBe3X37mIf3R6m4rFB4i0gb3IHCqN0DVxv3f91QGD24OnRQ8rZ9duP++1Ztg8TPw7lOw7o391yVFoPA4fOgk9hR8gD0ZRUSbakhrrCaloRpqq6CuKrgLbu3rLTcqkFkAw6bA0OPxhhoaKjdSv3MTXr2J5JoK0uq2UZ1awOaM0WyIjmR92kjWpY5kS1IBDU2N5O9ZQ8meZZTWLqes/j3KGlZSk5zFwO+sPKQ/HoUHCg8RiaOd62Hpc4AFD3YWHgeRaOf2dYet78Haf8Ca14MwqVxDSwsouxCyCiFrSDDlQNV62LQw2GfvRa9oXvCcUMPu4HNKevDcz9CJQfhNuCzmZofOU3ig8BCRI0jtTohkQvJBuqH3znC5aUEQJsmpQVgUTYRBow++byfpOQ8RkSNJNLfjbVIzgwE2e8nUyBpXQEREukzhISIiXabwEBGRLlN4iIhIlyk8RESkyxQeIiLSZQoPERHpMoWHiIh0WZ99wtzMKoA1h7j7IGBrN1bnSKHz7l903v1LZ877KHcv6MzB+mx4HA4zm9PZR/T7Ep13/6Lz7l+6+7x12UpERLpM4SEiIl2m8GjbrxJdgQTRefcvOu/+pVvPW30eIiLSZWp5iIhIlyk8YpjZOWa2zMxWmNktia5PPJnZb8xsi5m9G1OWb2Yvmtl74fuARNaxu5lZqZnNMrPFZrbIzG4My/v0eQOYWdTM/mlm74Tn/r2wfLiZvRn+zD9hZqmJrmt3M7NkM3vbzJ4LP/f5cwYws9VmttDM5pvZnLCs237WFR4hM0sG7gPOBcYCl5nZ2MTWKq5+B5zTquwW4CV3HwW8FH7uSxqBm919LDAFuD78O+7r5w1QB0x19wnAROAcM5sC/Ai4x91HAjuALyWwjvFyI7Ak5nN/OOe9PuLuE2Nu0e22n3WFxz4nASvcfZW71wOPAxcluE5x4+6vANtbFV8EPBwuPwxc3KOVijN33+ju88LlaoL/UIrp4+cN4IFd4cdI+HJgKvDHsLzPnbuZlQDnAw+Gn40+fs4d6LafdYXHPsXAupjP5WFZf1Lo7hvD5U1AYSIrE09mVgZMAt6kn5x3ePlmPrAFeBFYCVS6e2O4SV/8mf8p8O9Ac/h5IH3/nPdy4AUzm2tm14Rl3fazrjnMpU3u7mbWJ2/FM7Ms4EngJnevCn4ZDfTl83b3JmCimeUBfwLGJLhKcWVmFwBb3H2umZ2R6PokwKnuvt7MBgMvmtnS2JWH+7Oulsc+64HSmM8lYVl/stnMigDC9y0Jrk+3M7MIQXA86u5PhcV9/rxjuXslMAs4Gcgzs72/RPa1n/lTgAvNbDXBZeipwL307XNu4e7rw/ctBL8snEQ3/qwrPPZ5CxgV3omRClwKPJvgOvW0Z4Erw+UrgWcSWJduF17vfghY4u53x6zq0+cNYGYFYYsDM0sHziTo85kFfDrcrE+du7vf6u4l7l5G8O95prtfTh8+573MLNPMsvcuA2cB79KNP+t6SDCGmZ1HcI00GfiNu9+Z4CrFjZn9ATiDYKTNzcDtwNPAdGAYwYjEl7h76071I5aZnQq8Cixk3zXw2wj6PfrseQOY2QcIOkiTCX5pnO7u3zezEQS/lecDbwOfc/e6xNU0PsLLVv/m7hf0h3MOz/FP4ccU4DF3v9PMBtJNP+sKDxER6TJdthIRkS5TeIiISJcpPEREpMsUHiIi0mUKDxER6TKFh0gXmFlTOErp3le3DaJoZmWxoxyL9GYankSka/a4+8REV0Ik0dTyEOkG4dwJPw7nT/inmY0My8vMbKaZLTCzl8xsWFheaGZ/CufXeMfMPhQeKtnMfh3OufFC+DQ4ZnZDOA/JAjN7PEGnKdJC4SHSNemtLltNi1m3093HAz8nGKkA4L+Bh939A8CjwM/C8p8Bs8P5NY4HFoXlo4D73P04oBL4VFh+CzApPM6X43VyIp2lJ8xFusDMdrl7VhvlqwkmW1oVDr64yd0HmtlWoMjdG8Lyje4+yMwqgJLYYTHCYeJfDCfqwcy+CUTc/Q4z+xuwi2AImadj5uYQSQi1PES6j7ez3BWxYyw1sa9f8nyCmS6PB96KGRVWJCEUHiLdZ1rM++vh8j8IRnQFuJxgYEYIpgC9Flomacpt76BmlgSUuvss4JtALnBA60ekJ+m3F5GuSQ9n49vrb+6+93bdAWa2gKD1cFlY9lXgt2b2DaAC+GJYfiPwKzP7EkEL41pgI21LBv4nDBgDfhbOySGSMOrzEOkGYZ/HZHffmui6iPQEXbYSEZEuU8tDRES6TC0PERHpMoWHiIh0mcJDRES6TOEhIiJdpvAQEZEuU3iIiEiX/X9d4/SEDMNKigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc314b572d0>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize training performance\n",
    "history_df = pd.DataFrame(hist.history)\n",
    "hist_plot_file = os.path.join('figures', 'onehidden_vae_training.pdf')\n",
    "ax = history_df.plot()\n",
    "ax.set_xlabel('Epochs')\n",
    "ax.set_ylabel('VAE Loss')\n",
    "fig = ax.get_figure()\n",
    "fig.savefig(hist_plot_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
